package com.co.webmagic.process;

import com.co.common.util.IdWorker;
import com.co.video.pojo.Category;
import com.co.video.pojo.Detail;
import com.co.video.pojo.Video;
import com.co.video.pojo.VideoState;
import com.co.video.service.CategoryService;
import com.co.video.service.DetailService;
import com.co.video.service.VideoService;
import com.co.video.service.VideoStateService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Html;

import java.util.Date;
import java.util.List;
import java.util.Random;

/**
 * @author ldl.plus
 * @date 2020年06月30日  23:08
 */
@Component
public class OldJobProcessor implements PageProcessor {

    @Autowired
    private DetailService detailService;

    @Autowired
    private VideoService videoService;

    @Autowired
    private VideoStateService videoStateService;

    @Autowired
    private CategoryService categoryService;

    @Autowired
    private IdWorker idWorker;
    private static final Logger log = LoggerFactory.getLogger(OldJobProcessor.class);

    /**
     * 解析页面
     *
     * @param page
     */
    @Override
    public void process(Page page) {

        Html html = page.getHtml();
        String title = html.css("span.tit", "text").get();
        if (title != null) {
            Random random = new Random();
            List<String> categories = html.css("div.video-data span.a-crumbs>a", "text").all();
            String name = categories.get(0);
            Category category = categoryService.findByName(name);
            if (category == null) {
                category = new Category();
                category.setId(idWorker.nextId() + "");
                category.setName(name);
                category.setSeq(0);
                category.setParentId("");
                category.setNums(0);
                category.setCreateTime(new Date());
                category.setUpdateTime(new Date());
                categoryService.add(category);
            }
            category.setNums(category.getNums() + 1);
            categoryService.update(category);

            // 有这个视频，保存相关信息
            Detail detail = new Detail();
            detail.setCategoryId(category.getId());
            if (categories.size() > 1) {
                name = categories.get(1);
                Category category1 = categoryService.findByName(name);
                if (category1 == null) {
                    category1 = new Category();
                    category1.setId(idWorker.nextId() + "");
                    category1.setName(name);
                    category1.setSeq(0);
                    category1.setParentId(category.getId());
                    category1.setNums(0);
                    category1.setCreateTime(new Date());
                    category1.setUpdateTime(new Date());
                    categoryService.add(category1);
                }
                detail.setCategoryId(category1.getId());
                category1.setNums(category1.getNums() + 1);
                categoryService.update(category1);
            }

            detail.setAid(idWorker.nextId() + "");
            detail.setTitle(title);
            detail.setImage("");
            detail.setUrl(urlPre + "video/av" + count-- + "/");
            detail.setUploadTime(new Date());
            String totalTime = html.css("span.bilibili-player-video-time-total", "text").get();
            if (totalTime == null) {
                totalTime = "0";
            }
            detail.setDurations(Integer.parseInt(totalTime));
            detail.setStatus(1);
            String username = html.css("a.username", "text").get();
            detail.setUsername(username);
            detail.setIsUpload(1);
            String comment = html.css("div.v_desc div.info", "text").get();
            detail.setComment(comment);
            detail.setVideos(1);
            detail.setCreateTime(new Date());
            detail.setUpdateTime(new Date());

            Video video = new Video();
            video.setCid(idWorker.nextId() + "");
            video.setAid(detail.getAid());
            video.setPage(1);
            video.setFrom("crawler");
            video.setPart("1");
            video.setDuration(detail.getDurations());
            video.setUrl(urlPre + "video/av" + count-- + "/");
            video.setCreateTime(new Date());
            video.setUpdateTime(new Date());

            VideoState videoState = new VideoState();
            videoState.setId(idWorker.nextId() + "");
            videoState.setAid(detail.getAid());
            videoState.setView(random.nextInt(10000));
            videoState.setReply(random.nextInt(10000));
            videoState.setDanmaku(random.nextInt(10000));
            videoState.setFavorite(random.nextInt(10000));
            String like = html.css("span.like", "title").get().substring(3);
            videoState.setLike(Integer.parseInt(like));
            videoState.setDislike(0);
            videoState.setNowRank(0);
            videoState.setHisRank(0);
            videoState.setShare(random.nextInt(10000));
            videoState.setCreateTime(new Date());
            videoState.setUpdateTime(new Date());

            detailService.add(detail);
            videoService.add(video);
            videoStateService.add(videoState);

            log.info("--------------------------" + detail.getUrl() + "---------------------------");
        }

    }

    /**
     * 重试次数和抓取间隔
     */
    private final Site site = Site.me()
            .setCharset("UTF8")
            .setTimeOut(3000)
            .setRetrySleepTime(1000)
            .setRetryTimes(3)
            .setSleepTime(0)
            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36 Edg/83.0.478.56");

    @Override
    public Site getSite() {
        return site;
    }

    /**
     * 设置拼接的url变量
     */
    private static int count = 841227499;

    private static final String urlPre = "https://www.bilibili.com/";

    public static void main(String[] args) {
        while (count > 130000) {
            Spider.create(new OldJobProcessor())
                    .addUrl(urlPre + "video/av" + count-- + "/")
                    .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
                    .thread(1)
                    .run();
        }
    }
}
